"""

.. module:: build_datastore.py
.. moduleauthor:: @author: 

The purpose of this module is to build a datastore containing resources for all 
files within the argument Eclipse projects (eg. 'base'). 

All packages and sub-packages of the argument 'project' are read to determine
which packages should be included in the datastore. The rules for including
a package are:

1. ignore package if name starts with "."
2. ignore package if it does not contain a 'configuration.xml' file.

All files within each package are filtered to determine which files should be 
included in the datastore. The rules for including a file are: 

3. ignore file if filename starts with "."
4. ignore file if filename suffix is ".pyc"
5. ignore file if filename is "*test.py"

After filtering, the contents of all remaining files are written to the datastore
as Resources. The following rules determine whether the contents are deemed to be
serialised Resources (and written directly to the datastore), or a Resource is 
created containing the file contents:

6. if filename suffix is ".xml" then contents are deemed to be serialised Resource.
7. if filename suffix is ".html" then contents are deemed to be serialised Resource.
8. if filename suffix not ".xml" or ".html" then a Resource is created containing 
   the file contents.

Some website file types require a Resource Metadata key Uri be created to allow 
that file to be addressed from web pages. The rules are:

9. if filename suffix is ".js" then key = 'http://localhost:8000/<project>/js/<filename>.js'
10. if filename suffix is ".css" then key = 'http://localhost:8000/<project>/css/<filename>.css'
11. if filename suffix is ".jpg" then key = 'http://localhost:8000/<project>/i/<filename>.jpg'

Options for the build are:

--upload 
    - upload datastore to Google code.

--gc_username
    - Google Code user name.

--gc_password
    - Google Code password.

--gc_summary
    - Description of datastore to upload to Google Code.

--gc_labels
    - Labels for datastore in Google Code

--project 
    - the project to put into datastore.

--version 
    - the version of the project to put into the datastore. If multiple 
      projects are specified then they must each be tagged with the same
      version. If no version is specified then the current workspace code
      is used. The output datastore will be named with this version.
      
--workspace 
    - the root directory of the Eclipse workspace.
    
--distribution_directory 
    - the directory where the datastore is placed.

--datastore_name
    - the name of the datastore. This name will be appended with the version 
      tag and "datastore" suffix.
       
.. seealso:: modules :mod:`<module>`

*Constants*:

.. data:: CONFIGURATION_XML
        
    CONFIGURATION_XML is the name of the configuration XML file - 'configuration.xml'
"""

# Imports
#
import base64
import os
import sys
import pysvn
# this piece of code actually adds the src dir to the PYTHONPATH. Hence if this script is invoked from command line,
# the user does not have to setup the PYTHONPATH. This will resolve import errors if the PYTHONPATH is not set up correctly.
# Hence the order of the imports are IMPORTANT. Make sure that no other import other an 'import sys' goes above the statement below 

from datastore.database import DataStore
from datastore.resource import Resource
from datastore.resource import DeserialisationException
from datastore.uri import Uri

# TODO: add simplified imports for package and runtime.
from package import googlecode_upload
from optparse import OptionParser

# Constants
#
CONFIGURATION_XML = 'configuration.xml'

# Classes
#

def create_resource_from_file(project, package_pathname, module_pathname, file_name, content):
    """This function creates a Resource object from the argument content.
    
    :param project: String - the project name
    :param package_pathname: String - the name of the package
    :param module_pathname: String - the name of the module
    :param file_name: String - the pathname of the file
    :param content: String - the file contents
    :rtype: Resource
    """
    
    # do not create resource when content is empty. This is because
    # python does not like exec'ing empty content
    #
    if (not content) or (not content.strip()):
        return None
    
    rs = Resource()
    rs.content = base64.b64encode(content)
    
    # 9. if filename suffix is ".js" then key = 'http://localhost:8000/<project>/js/<filename>.js'
    # 10. if filename suffix is ".css" then key = 'http://localhost:8000/<project>/css/<filename>.css'
    # 11. if filename suffix is ".jpg" then key = 'http://localhost:8000/<project>/i/<filename>.jpg'
    #
    
    resourcetype_value_rs = None
    http_uri_rs = Uri(Resource.BINDING_NS_URI_LOCALHOST)
    
    if (file_name.endswith('.js')):
        resourcetype_value_rs = Resource.KEY_URI_PACKAGE_TYPE_DATA
        http_uri_rs = http_uri_rs + project + 'js' + file_name 
        rs.set_metadata(Resource.KEY_URI_PACKAGE_DATAURI, http_uri_rs)
    elif (file_name.endswith('.css')):
        resourcetype_value_rs = Resource.KEY_URI_PACKAGE_TYPE_DATA
        http_uri_rs = http_uri_rs + project + 'css' + file_name 
        rs.set_metadata(Resource.KEY_URI_PACKAGE_DATAURI, http_uri_rs)
    elif (file_name.endswith('.jpg')):
        resourcetype_value_rs = Resource.KEY_URI_PACKAGE_TYPE_DATA
        http_uri_rs = http_uri_rs + project + 'i' + file_name 
        rs.set_metadata(Resource.KEY_URI_PACKAGE_DATAURI, http_uri_rs)
    else:
        resourcetype_value_rs = Resource.KEY_URI_PACKAGE_TYPE_MODULE
    
    rs.set_metadata(Resource.KEY_URI_PACKAGE_FILENAME, file_name)
    rs.set_metadata(Resource.KEY_URI_PACKAGE_MODULENAME, module_pathname)
    rs.set_metadata(Resource.KEY_URI_PACKAGE_PACKAGENAME, package_pathname)
    rs.set_metadata(Resource.KEY_URI_PACKAGE_PROJECTNAME, project)
    rs.set_type(resourcetype_value_rs)
    
    return rs

def process_package_directory(dataStore, project, package_directory, package_pathname):
    """This function processes the contents of the argument package directory.
    
    :param dataStore: database.dataStore - the datastore to write Resource to. 
    :param project: String - the project name 
    :param package_directory: String - the pathname of the package 
    :param package_pathname: String - the package path 
    """
    
    print "Processing package (%s)(%s)..." % (package_directory, package_pathname)
    
    files = os.listdir(package_directory)
    for f in files:
        file_pathname = os.path.join(package_directory, f)
        
        # 1. ignore package if name starts with "."
        # 2. ignore file if filename starts with "."
        #
        if (not f.startswith('.')):
            # file may be a sub-directory. If so, recurse...
            #
            if (os.path.isdir(file_pathname)):
                pkg_pathname = ''.join([package_pathname, ".", os.path.basename(file_pathname)])
                process_package_directory(dataStore, project, file_pathname, pkg_pathname)
            else:
                # 3. ignore file if filename starts with "." (done above)
                # 4. ignore file if filename suffix is ".pyc"
                # 5. ignore file if filename is "*test.py"
                #
                if (not f.endswith('.pyc')) and (not f.endswith('test.py')):
                    print "Adding file (%s)..." % file_pathname
                    
                    file = open(file_pathname, 'r')
                    content = file.read()
                    rs = None
                    
                    # 6. if filename suffix is ".xml" then contents are deemed to be serialised Resource.
                    # 7. if filename suffix is ".html" then contents are deemed to be serialised Resource.
                    # 8. if filename suffix not ".xml" or ".html" then a Resource is created containing the file contents.
                    #
                    if (f.endswith('.xml')) or (f.endswith('.html')):
                        try:
                            rs = Resource(content)
                        except DeserialisationException as d:
                            print "File (%s) is not a valid deserialised resource. Exception (%s)" % (file_pathname, d), 
                            sys.exit(1)
                    else:
                        # make sure that we consistently use \n as the line ending.
                        #
#                        content = content.replace('\r\n', '\n')
                        
                        module_pathname = ""
                        
                        # strip the characters after, and including, "."
                        #
                        dotPos = f.find(".")
                        if (dotPos > -1):
                            module_pathname = ''.join([package_pathname, ".", f[0:dotPos]])
                        else:
                            module_pathname = ''.join([package_pathname, ".", f])
                            
                        rs = create_resource_from_file(project, package_pathname, module_pathname, f, content)
                    
                    if (rs):
                        dataStore.put(rs)
                    
                    file.close()
                    file = None
                else:
                    print "Ignoring file (%s)..." % file_pathname
        else:
            print "Ignoring file/directory (%s)..." % file_pathname
        
if __name__ == '__main__':
    
    # FIXME: remove the need for a specific current directory . 
    parser = OptionParser(usage=u"%s options. (Note: must be in the build_datastore.py source directory.)" % sys.argv[0])

    parser.add_option("--upload", dest=u"do_upload_to_googlecode", action="store_true", default=False, help=u"Upload datastore to Google Code.")
    parser.add_option("--gc_username", dest=u"gc_username", help=u"Google Code username")
    parser.add_option("--gc_password", dest=u"gc_password", help=u"Google Code password")
    parser.add_option("--gc_summary", dest=u"gc_summary", help=u"Description of datastore to upload to Google Code")
    parser.add_option("--gc_labels", dest=u"gc_labels", help=u"Labels for datastore in Google Code")
    parser.add_option("-p", "--project", dest=u"project_names", action="append", help=u"Name of the Eclipse project to put into datastore. e.g base")
    parser.add_option("-v", "--version", dest=u"version_tag", help=u"Version of the project to build. If supplied, this must match an SVN tag. If not specified, the current workspace code is used.")
    parser.add_option("-w", "--workspace", dest=u"workspace_directory", help=u"Absolute Path to the Eclipse workspace directory.")
    parser.add_option("-b", "--distribution_directory", dest=u"distribution_directory", help=u"Absolute Path to the directory where the datastore is created.")    
    parser.add_option("-d", "--datastore_name", dest=u"datastore_name", help=u"Name of the datastore. This name is appended with '.datastore' suffix.")    

    (options, args) = parser.parse_args()
    options.script = os.path.split(sys.argv[0])[1]
    
    if (options.do_upload_to_googlecode is True):
        if (options.gc_username is None) or (options.gc_password is None):
            parser.error("Must specify Google Code username and password to upload datastore to Google Code.")
        
        if (options.gc_summary is None) or (options.gc_labels is None):
            parser.error("Must specify summary and labels to upload datastore to Google Code.")
        
        options.gc_labels = options.gc_labels.split(',')
        
    if (options.project_names is None):
        parser.error("No project specified.")
    
    if (options.version_tag is None):
        print("No version specified. Using current workspace.")
    
    if (options.workspace_directory is None):
        parser.error("No workspace directory specified.")
    
    if (options.distribution_directory is None):
        parser.error("No build directory specified.")
    
    if (options.datastore_name is None):
        parser.error("No datastore name specified.")
    
    print "Upload to Google Code (%s)" % options.do_upload_to_googlecode
    if (options.do_upload_to_googlecode is True):
        print "Google Code username (%s)" % options.gc_username
        print "Google Code password (%s)" % options.gc_password
        print "Google Code summary (%s)" % options.gc_summary
        print "Google Code labels (%s)" % options.gc_labels
    print "Project Name (%s)" % options.project_names
    print "Version (%s)" % options.version_tag
    print "Workspace Directory (%s)" % options.workspace_directory
    print "Distribution Directory (%s)" % options.distribution_directory
    print "Datastore Name (%s)" % options.datastore_name
    
    # Checkout code from SVN to the build directory, but only if a version tag was specified.
    # We must perform this checkout before checking if the project exists.
    #
    if (options.version_tag is not None):
        print("Checking out code from repository")
        client = pysvn.Client()
        
        # TODO: Checkout code from SVN using the tag.
        #
        
        client.checkout("http://yarra.googlecode.com/svn/trunk", options.distribution_directory)
        
        code_root_directory = options.distribution_directory
    else:
        code_root_directory = options.workspace_directory
    
    print "Code Root Directory (%s)" % code_root_directory
    
    # check if projects exist. 
    #
    for project in options.project_names:
        project_root_directory = os.path.join(code_root_directory, project, "src") 
        if (not os.path.exists(project_root_directory)):
            print "Cannot find project name (%s)" % project
            sys.exit(-1)
        
        print "Project Root Directory (%s)" % project_root_directory

    # Delete the database if it already exists and create new DataStore.
    #
    datastore_pathname = os.path.join(options.distribution_directory, options.datastore_name + ".datastore")
    if (os.path.exists(datastore_pathname)):
        os.remove(datastore_pathname)
    
    print "Datastore Path (%s)" % datastore_pathname
    
    dataStore = DataStore(datastore_pathname)
    
    for project in options.project_names:
        print "Adding project (%s)..." % project
        
        project_root_directory = os.path.join(code_root_directory, project, "src") 
        
        # Find all packages in the project.
        #
        for package_name in os.listdir(project_root_directory):
            package_directory = os.path.join(project_root_directory, package_name)
            
            # 1. ignore package if name starts with "."
            #    - ignore plain files and "." directories, including ".svn"
            #
            if (os.path.isdir(package_directory)) and (not package_name.startswith(".")):
                # 2. ignore package if it does not contain a 'configuration.xml' file.
                #
                manifest_file_path = os.path.join(package_directory, CONFIGURATION_XML)
                if (not os.path.exists(manifest_file_path)):
                    print "No 'configuration.xml' file in package (%s). Ignoring ..." % package_directory
                else:
                    process_package_directory(dataStore, project, package_directory, package_name)
            else:
                print "Ignoring package (%s)..." % package_directory

    # upload to Google Code
    #
    if (options.do_upload_to_googlecode is True):
        print('Uploading to Google Code...')
        
        http_code, http_reason, file_url = googlecode_upload.upload(dataStore.get_datastore_pathname(), 'yarra', options.gc_username, options.gc_password, options.gc_summary, options.gc_labels)
        
        if (http_code == 201):
            print "Build uploaded successfully."
        else:
            print "Build Failed. HTTP Status (%s), Reason (%s)" % (http_code, http_reason) 